# -*- coding: utf-8 -*-
import re,os,sqlite3,urllib2,datetime,zlib,gzip,struct,pickle,base64
# 310120110517964514
# h='31012011051'
# 3101表示宁波海关、3104表示北仑海关，报关单号开头表示在哪个海关报关
c1='3101'
# 表示年份
y='2011'
# 0表示出口，1表示进口
i='0'
# 0出口时：
# 51表示宁波海关，54表示北仑海关
# 1进口时：
# 01表示宁波海关，04表示北仑海关
c2='51'
h=c1+y+i+c2
s=7964514
e=7964516
global proc
proc=0
total=float(e-s-1)

conn=sqlite3.connect(r'html.db3')
conn.text_factory = str
# db=bsddb.btopen('html.db','c')
# conn.text_factory = sqlite3.OptimizedUnicode
# c = conn.cursor()
def dbcreate():

    try:
        conn.execute('''CREATE TABLE html(
            cd_no INTEGER PRIMARY KEY,
            cd_html TEXT
        )''')

        conn.commit()
    except:
        pass
def dbfind(tbl,fld,val):
    return conn.execute('select id from '+tbl+' where '+fld+'="'+val+'"').fetchone()[0]
def dbread(cd_no):
    return conn.execute('select cd_html from html where cd_no=?',(cd_no,)).fetchone()[0]
def dbinsert(cd_no,html):
    # 插入海关，如果id存在就覆盖
    conn.execute('replace into html values(?,?)',(cd_no,html))
def gethtml(cd_no,sign=1):
    url='http://www.nbedi.com/h2000eport_ic/pre_bgd/EntryLogDataSearch/bgd_print/normal_print.asp?sign='+str(sign)+r'&pre_entry_id='+str(cd_no)
    # print url
    content=urllib2.urlopen(url).read()
    if(sign!=1):
        f=open(cd_no+r'.'+str(sign),'w')
    else:
        f=open(cd_no,'w')
    f.write(content)
    f.close()
    return content
dbcreate()
starttime=datetime.datetime.now()
for x in xrange(s,e):
    curr=int(((x-s)*100/total))
    if curr<>proc:
        proc=curr
        currtime=datetime.datetime.now()
        print proc,'%',currtime.time(),currtime-starttime
        
    sign=1
    fname=h+str(x)
    if(os.path.isfile(fname)):
        f=open(fname)
        html=f.read()
        f.close()
    else:
        continue
        # html=gethtml(fname)
    # 报关单号是否存在
    try:
        re.search(fname,html).group(0)
    except:
        os.remove(fname)
        # continue
    # 是否是样本
    # pattern=unicode(r"<font size='4'>样&nbsp;&nbsp;本</font>",'utf8')
    # sample=re.search(pattern,html.decode('gbk'))
    pattern=r"<font size='4'>.+?</font>"
    sample=re.search(pattern,html)
    if(sample!=None):
        # fs=open('sample.txt','a')
        # fs.write(fname+'\n')
        # fs.close()
        # continue
        os.remove(fname)
    
    pattern='\s+'
    html=re.sub(pattern,' ',html)
    f=open(fname+r'.htm','w')
    f.write(html)
    f.close()
    pattern='> <'
    html=re.sub(pattern,'><',html)
    f=open(fname+r'.html','w')
    f.write(html)
    f.close()
    pattern='<tr><td valign="top"><table width="99%" height="44" border="0" align="right" cellpadding="0" cellspacing="0"><tr><td></td></tr></table></td></tr>|<object.+?</object>'
    html=re.sub(pattern,'',html)
    f=open(fname+r'.t','w')
    f.write(html)
    f.close()
    z=zlib.compress(html,9)
    bz=base64.b64encode(z)
    b=base64.b64encode(html)
    print b
    f=open(fname+r'.zlib','wb')
    f.write(z)
    f.close()
    f=open(fname+r'.bz64','wb')
    f.write(b)
    f.close()
    f=open(fname+r'.b64','wb')
    f.write(b)
    f.close()
    # print '%r' %z
    # print pickle.dumps(z)
    
    # dbinsert(fname,html)
    # conn.execute('replace into html values(?,?)',(fname,sqlite3.Binary(z)))
    # a=dbread(fname)
    # a=conn.execute('select z from html where cd_no=?',(fname,)).fetchone()[0]
    # z=zlib.decompress(a)
    # f=open(fname+r'.dz','w')
    # f.write(z)
    # print z
    # f.close()
    # for i in xrange(100):
        # db[str(i)]=b
    # print zlib.decompress(base64.b64decode(db['2']))
    
        
conn.commit()
# conn_edi.commit()
conn.close()
# conn_edi.close()
